/******************************************************************************

	COPYRIGHT(C) JONAS 'SORTIE' TERMANSEN 2010.

	This file is part of Maxsi Distribution.

	Maxsi Distribution is free software: you can redistribute it and/or modify
	it under the terms of the GNU Lesser General Public License as published by
	the Free Software Foundation, either version 3 of the License, or (at your
	option) any later version.

	Maxsi Distribution is distributed in the hope that it will be useful, but
	WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
	or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
	License for more details.

	You should have received a copy of the GNU Lesser General Public License
	along with Maxsi Distribution. If not, see <http://www.gnu.org/licenses/>.

	Maxsi Distribution
	A powerful digital distribution library designed for games and software.

	MD_Release.cpp
	A system that allows you to release product data to master servers and
	several content servers.

******************************************************************************/

#include "MaxsiLibrary.h"
#include "MaxsiDistribution.h"

#define BLOCK_SIZE (4*1024*1024)
#define SHA1_BUFFER_SIZE (16*1024)

Word_t ThreadReleaseProduct(BYTE* Parameter)
{
	((MD_Release*)Parameter)->ReleaseProduct();

	return 0;
}

Word_t ThreadReleaseProductSHA(BYTE* Parameter)
{
	((MD_Release*)Parameter)->ReleaseProductSHA();

	return 0;
}

Word_t ThreadReleaseProductUpload(BYTE* Parameter)
{
	((MD_Release*)Parameter)->ReleaseProductUpload();

	return 0;
}

size_t UploadCallback(BYTE* UserData, IHTTP* Context, BYTE* Buffer, size_t Available)
{
	// WARNING: Buffer is NOT always null-terminated at Buffer[Available]!
	PrintOutput("%s", Buffer);

	return 0;
}

MD_Release::MD_Release()
{
	Callback			=	NULL;
	UserData			=	NULL;
	ContentArchive		=	NULL;
	MasterServers		=	NULL;
	NumMasterServers	=	0;
	ContentServers		=	NULL;
	NumContentServers	=	0;
	MasterImagePath		=	NULL;
	Caller				=	NULL;
	FileList			=	NULL;
	BlockHashList		=	NULL;
	FileHashList		=	NULL;
	TotalFileSize		=	0;
	BlockSize			=	BLOCK_SIZE;
	NumBlocks			=	0;
	FileListString		=	NULL;
}

MD_Release::~MD_Release()
{
	IFDEL(ContentArchive);
	IFDEL(MasterImagePath);
	IFDEL(FileListString);

	if ( MasterServers ) { DeleteStringList(MasterServers, NumMasterServers); }
	if ( ContentServers ) { DeleteStringList(ContentServers, NumContentServers); }

	if ( FileList ) { DeleteList(FileList); }
	if ( FileHashList ) { DeleteList(FileHashList); }
	if ( BlockHashList ) { DeleteList(BlockHashList); }
}

void MD_Release::SignalProgress(int Event, int Parameter, float Progress)
{
	if ( Callback ) { Callback(UserData, Caller, (MaxsiHandle)this, Event, Parameter, Progress); }
}

MaxsiError MD_Release::ScanDir(MESTR* AbsPath, MESTR* RelPath)
{
	SignalProgress(MD_RELEASE_EVENT_SCAN, MAXSI_PROGRESS_INDETERMINATE);

	MaxsiDirectoryEntry	Folder;

	MaxsiError	Result		=	FileSystem()->OpenFolder(AbsPath, &Folder);

	if ( Result != MAXSI_ERROR_SUCCESS ) { return Result; }

	while ( Result == MAXSI_ERROR_SUCCESS && FileSystem()->ReadFolder(&Folder) == MAXSI_ERROR_SUCCESS )
	{
		if ( MESTRCMP(Folder.FileName, _MESTR(".")) == 0 || MESTRCMP(Folder.FileName, _MESTR("..")) == 0 ) { continue; }

		MESTR* RelFilePath	=	BuildString(2, RelPath, Folder.FileName);

		if ( RelFilePath == NULL ) { Result = MAXSI_ERROR_MEM_ALLOC_FAIL; }

		if ( Folder.Type == MAXSI_DIR_ENTRY_UNKNOWN )
		{
			//PrintOutput("Ignoring unknown %MS (what is this?)\n", RelFilePath);	
		}
		else if ( Folder.Type == MAXSI_DIR_ENTRY_DIR )
		{
			// This is a recursive function! Let's let the stack play chicken with the filesystem!
			//PrintOutput("Dir\t%MS/\n", RelFilePath);

			MESTR* NewRelPath		=	BuildString(2, RelFilePath, _MESTR("/"));
			MESTR* NewAbsPath		=	BuildString(3, AbsPath, Folder.FileName, _MESTR("/"));

			if ( NewRelPath == NULL || NewAbsPath == NULL )
			{
				Result = MAXSI_ERROR_MEM_ALLOC_FAIL; 
			}
			else
			{
				ScanDir(NewAbsPath, NewRelPath);
			}

			if ( NewRelPath != NULL ) { delete[] NewRelPath; }
			if ( NewAbsPath != NULL ) { delete[] NewAbsPath; }
		}
		else if ( Folder.Type == MAXSI_DIR_ENTRY_FILE )
		{
			MESTR* NewAbsPath		=	BuildString(2, AbsPath, Folder.FileName);
	
			if ( NewAbsPath == NULL )
			{
				Result = MAXSI_ERROR_MEM_ALLOC_FAIL;
			}
			else
			{
				MaxsiHandle FileHandle;
				if ( (FileHandle =FileSystem()->OpenFile(NewAbsPath, 0) ) != NULL )
				{
					FileSize_t FileSize		=	FileSystem()->GetFileSize(FileHandle);

					if ( AddListEntryM(&FileList, NewAbsPath) == false || AddListEntryM(&FileList, RelFilePath) == false || AddListEntry(&FileList, (BYTE*)&FileSize, sizeof(FileSize)) == false )
					{
						Result = MAXSI_ERROR_MEM_ALLOC_FAIL;				
					}
					else
					{
						TotalFileSize  += FileSize;
					}

					FileSystem()->CloseFile(FileHandle);
				}
				else
				{
					//PrintOutput("Ignoring file %MS (could not read)\n", RelFilePath);
				}
			}

		}
		else
		{
			//PrintOutput("Ignoring other %MS (what is this?)\n", RelFilePath);	
		}

		delete[] RelFilePath;
	}

	FileSystem()->CloseFolder(&Folder);

	SignalProgress(MD_RELEASE_EVENT_SCAN, MAXSI_PROGRESS_NORMAL, 1.0f);

	return Result;
}

MaxsiError MD_Release::CalculateFileHashes()
{
	MESTR* TotalFileSizeString	=	PrintFileSizeShort(TotalFileSize);

	if ( TotalFileSizeString == NULL ) { return MAXSI_ERROR_MEM_ALLOC_FAIL; }	

	size_t NumFileListEntries = GetListSize(FileList);

	size_t CurrentDot		=	0;
	size_t DotsPrinted		=	0;
	size_t NumDots			=	100;

	size_t BlockLeft		=	BlockSize;

	fflush(stdout);

	delete[] TotalFileSizeString;

	FileSize_t		AmountHashed	=	0;
	MaxsiHandle		FileHandle		=	NULL;

	BYTE* 	HashBuffer		=	new BYTE[SHA1_BUFFER_SIZE];

	if ( HashBuffer == NULL )  { return MAXSI_ERROR_MEM_ALLOC_FAIL; }

	MaxsiHandle	FileSHA		=	CreateSHA1Context();
	MaxsiHandle	BlockSHA	=	CreateSHA1Context();

	for ( size_t I = 0; I < NumFileListEntries; I += 3 )
	{
		MESTR*	AbsPath		=	GetListEntryM(FileList, I);
		//MESTR*	RelPath		=	GetListEntryM(FileList, I+1);
		
		if ( (FileHandle = FileSystem()->OpenFile(AbsPath, MAXSI_FILE_READ | MAXSI_FILE_SEQUENTIAL)) != NULL )
		{
			FileSize_t FileSize		=	FileSystem()->GetFileSize(FileHandle);
			FileSize_t FileLeft		=	FileSize;

			while ( FileLeft )
			{
				size_t FragmentSize		=	(FileLeft > SHA1_BUFFER_SIZE) ? SHA1_BUFFER_SIZE : FileLeft;
				size_t FragmentLeft		=	FragmentSize;

				FileSystem()->ReadFile(FileHandle, HashBuffer, FragmentSize);

				// Update the file hash
				UpdateSHA1Context(FileSHA, HashBuffer, FragmentSize);

				size_t FragmentBlockRead	=	0;

				// Update the block hashes
				while ( FragmentLeft )
				{
					size_t BlockFragmentSize	=	FragmentLeft;

					if ( BlockFragmentSize > BlockLeft ) { BlockFragmentSize = BlockLeft; }

					UpdateSHA1Context(BlockSHA, HashBuffer + FragmentBlockRead, BlockFragmentSize);

					BlockLeft			-=	BlockFragmentSize;
					FragmentLeft		-=	BlockFragmentSize;
					FragmentBlockRead	+=	BlockFragmentSize;

					if ( BlockLeft == 0 )
					{
						char* BlockHash	=	ReportSHA1ContextA(BlockSHA);

						AddListEntryA(&BlockHashList, BlockHash);

						ResetSHA1Context(BlockSHA);
						BlockLeft	=	BlockSize;
					}
				}

				AmountHashed	+=	FragmentSize;
				FileLeft		-=	FragmentSize;

				// Calculate the progress and display it.
				CurrentDot	=	(NumDots * AmountHashed) / TotalFileSize;

				if ( CurrentDot > DotsPrinted )
				{
					SignalProgress(MD_RELEASE_EVENT_SHA1, MAXSI_PROGRESS_NORMAL, (DotsPrinted / 100.0f));

					DotsPrinted		=	CurrentDot;
					fflush(stdout);
				}
			}

			FileSystem()->CloseFile(FileHandle);

			char* FileHash	=	ReportSHA1ContextA(FileSHA);

			AddListEntryA(&FileHashList, FileHash);

			ResetSHA1Context(FileSHA);
		}
	}
	
	if ( FileSHA != NULL ) { DeleteSHA1Context(FileSHA); }
	if ( BlockSHA != NULL ) { DeleteSHA1Context(BlockSHA); }

	SignalProgress(MD_RELEASE_EVENT_SHA1, MAXSI_PROGRESS_NORMAL, 1.0f);

	return MAXSI_ERROR_SUCCESS;
}

MaxsiError MD_Release::CalculateFileList()
{
	SignalProgress(MD_RELEASE_EVENT_MASTERING, MAXSI_PROGRESS_NORMAL, 0.5f);

	// First convert all the file names to UTF-8.
	
	size_t NumEntries	=	GetListSize(FileList);

	List UTF8FileList	=	CreateList(NumEntries / 3);

	if ( UTF8FileList == NULL ) { return MAXSI_ERROR_MEM_ALLOC_FAIL; }

	for ( size_t I = 0; I < NumEntries; I += 3 )
	{
		UTF8_t*		UTF8FileName	=	MESTR2UTF8(GetListEntryM(FileList, I + 1));

		if ( UTF8FileName == NULL || SetListEntryA(UTF8FileList, I/3, (char*)UTF8FileName) == false ) { DeleteList(UTF8FileList); return MAXSI_ERROR_MEM_ALLOC_FAIL; }
	}

	// A static memory buffer large enough to hold the largest filesize in base 10.
	UTF8_t	FileSizeString[FILESIZE_MAX_LEN_BASE_10 + 1];

	// Create a buffer assembler that is able to combine all the input we will be giving it.
	BufferAssembler		Assembler		=	CreateBufferAssembler();

	if ( Assembler == NULL ) { DeleteList(UTF8FileList); }	

	// Now that we have a glorious list of UTF-8 encoded filenames, go ahead and burn a bufferful of filenames, filesizes, blocks, and hashes!
	
	for ( size_t I = 0; I < NumEntries; I += 3 )
	{
		// Find the filename of the file.
		UTF8_t*		FileName	=	(UTF8_t*)GetListEntryA(UTF8FileList, I / 3);

		// Find the filesize of the file.
		FileSize_t*	FileSize	=	(FileSize_t*)GetListEntryA(FileList, I + 2);

		// Print the filesize of the file.
		FileSizeString[PrintUInt64A10((char*)FileSizeString, *FileSize)] = 0;

		// Find the SHA-1 hash of the file.
		char*		FileHash	=	GetListEntryA(FileHashList, I/3);

		// Now write data to our buffer assembler!
		WriteBufferDataA(Assembler, "File \"");
		WriteBufferDataU(Assembler, FileName);
		WriteBufferDataA(Assembler, "\" ");
		WriteBufferDataU(Assembler, FileSizeString);
		WriteBufferDataA(Assembler, " \"");
		WriteBufferDataA(Assembler, FileHash);
		WriteBufferDataA(Assembler, "\"\r\n");
	}

	for ( size_t I = 0; I < NumBlocks; I++ )
	{
		// Find the SHA-1 hash of the file.
		char*		BlockHash	=	GetListEntryA(BlockHashList, I);

		// Now write data to our buffer assembler!
		WriteBufferDataA(Assembler, "Block \"");
		WriteBufferDataA(Assembler, BlockHash);
		WriteBufferDataA(Assembler, "\"\r\n");
	}

	FileListString	=	AssembleBuffer(Assembler);

	// Clean up!
	DeleteBufferAssembler(Assembler);
	DeleteList(UTF8FileList);

	return MAXSI_ERROR_SUCCESS;
}

MaxsiError MD_Release::UploadFileList()
{
	SignalProgress(MD_RELEASE_EVENT_MASTERING, MAXSI_PROGRESS_INDETERMINATE);

	for ( size_t I = 0; I < NumMasterServers; I++ )
	{
		char*	MasterServer			=	MESTR2CHAR(MasterServers[I]);

		if ( MasterServer == NULL ) { return GetAnError(MAXSI_ERROR_MEM_ALLOC_FAIL); }

		// Create a MIME attachment containing the block!
		size_t MyMIMESize;
		BYTE* MyMIME = CreateHTTPMIMEBodyFileAttachment("TheBlock", (BYTE*)FileListString, strlen((char*)FileListString), &MyMIMESize);

		if ( MyMIME == NULL ) { delete[] MasterServer; return GetAnError(MAXSI_ERROR_MEM_ALLOC_FAIL); }

		char* URL	=	PrintStringA("%s?action=putfilelist&ContentArchive=at10", MasterServer);

		if ( URL == NULL ) { delete[] MasterServer; delete[] MyMIME; return GetAnError(MAXSI_ERROR_MEM_ALLOC_FAIL); }

		MAXSI_TODO("Check if any of the follow function fails!");

		IHTTP*	HTTP		=	(IHTTP*)CreateInterface("IHTTP");
		HTTP->SetCallback(UploadCallback);
		HTTP->SetCallbackUserData(NULL);
		HTTP->SetVersion(1, 1);
		HTTP->SetMethod((ASCII_t*)"POST");
		HTTP->SetURLA(URL);
		HTTP->SetMimeAttachmentBody(MyMIME, MyMIMESize);

		if ( HTTP->Continue() != MAXSI_ERROR_SUCCESS )
		{
			//PrintOutput("%MS %zu\n", Translate("MaxsiRelease/CouldNotUploadDataBlock"), BlockId);
		}

		//PrintOutput("\n");

		DeleteInterface(HTTP);

		delete[] MyMIME;
		delete[] MasterServer;
	}

	SignalProgress(MD_RELEASE_EVENT_MASTERING, MAXSI_PROGRESS_NORMAL, 1.0f);

	return MAXSI_ERROR_SUCCESS;
}

MaxsiError MD_Release::UploadBlocks()
{
	MESTR* TotalFileSizeString	=	PrintFileSizeShort(TotalFileSize);

	if ( TotalFileSizeString == NULL ) { return MAXSI_ERROR_MEM_ALLOC_FAIL; }	

	size_t NumFileListEntries = GetListSize(FileList);

	size_t BlockLeft		=	BlockSize;
	size_t BlockId			=	0;

	delete[] TotalFileSizeString;

	MaxsiHandle		FileHandle		=	NULL;

	MaxsiCompressionAlgorithmLZMA*	LZMA	=	(MaxsiCompressionAlgorithmLZMA*)CreateInterface("MaxsiCompressionAlgorithmLZMA");

	size_t	CompressedBufferSize	=	LZMA->GetCompressedBound(BlockSize);

	BYTE* 	UncompressedBuffer		=	new BYTE[BlockSize];
	BYTE* 	CompressedBuffer		=	new BYTE[CompressedBufferSize];

	for ( size_t N = 0; N < NumContentServers; N++ )	
	{
		char*	ContentServer			=	MESTR2CHAR(ContentServers[N]);

		for ( size_t I = 0; I < NumFileListEntries; I += 3 )
		{
			MESTR*	AbsPath		=	GetListEntryM(FileList, I);
			//MESTR*	RelPath		=	GetListEntryM(FileList, I+1);
		
			if ( (FileHandle = FileSystem()->OpenFile(AbsPath, MAXSI_FILE_READ | MAXSI_FILE_SEQUENTIAL)) != NULL )
			{
				FileSize_t FileSize		=	FileSystem()->GetFileSize(FileHandle);
				FileSize_t FileLeft		=	FileSize;

				while ( FileLeft )
				{
					size_t FragmentSize		=	(FileLeft > BlockLeft) ? BlockLeft : FileLeft;

					FileSystem()->ReadFile(FileHandle, UncompressedBuffer + (BlockSize - BlockLeft), FragmentSize);
				
					FileLeft		-=	FragmentSize;
					BlockLeft		-=	FragmentSize;

					if ( ( BlockLeft == 0 || I == NumFileListEntries - 2 ) && BlockLeft != BlockSize )
					{
				
						// Compress the block!
						PrintOutput("Compressing Block\t%zu/%zu", BlockId, NumBlocks);
						fflush(stdout);

						SignalProgress(MD_RELEASE_EVENT_COMPRESS, MAXSI_PROGRESS_NORMAL, (float)BlockId/(float)NumBlocks);
				
						size_t CompressedSize	=	CompressedBufferSize;
						LZMA->Compress(9, UncompressedBuffer, (BlockSize - BlockLeft), CompressedBuffer, &CompressedSize);

						PrintOutput("\nUploading Block\t\t%zu/%zu", BlockId, NumBlocks);
						fflush(stdout);						

						SignalProgress(MD_RELEASE_EVENT_UPLOAD, MAXSI_PROGRESS_NORMAL, (float)BlockId/(float)NumBlocks);
	
						// Create a MIME attachment containing the block!
						size_t MyMIMESize;
						BYTE* MyMIME = CreateHTTPMIMEBodyFileAttachment("TheBlock", (BYTE*)CompressedBuffer, CompressedSize, &MyMIMESize);

						char* URL	=	PrintStringA("%s?action=upload&ContentArchive=at10&BlockId=%zu", ContentServer, BlockId, ( BlockId != 0 ));

						MAXSI_TODO("Check if any of the follow function fails!");

						IHTTP*	HTTP		=	(IHTTP*)CreateInterface("IHTTP");
						HTTP->SetCallback(UploadCallback);
						HTTP->SetCallbackUserData(NULL);
						HTTP->SetVersion(1, 1);
						HTTP->SetMethod((ASCII_t*)"POST");
						HTTP->SetURLA(URL);
						HTTP->SetMimeAttachmentBody(MyMIME, MyMIMESize);

						if ( HTTP->Continue() != MAXSI_ERROR_SUCCESS )
						{
							//PrintOutput("%MS %zu\n", Translate("MaxsiRelease/CouldNotUploadDataBlock"), BlockId);
						}

						PrintOutput("\n");

						DeleteInterface(HTTP);

						delete[] MyMIME;

						BlockId++;
			
						BlockLeft	=	BlockSize;		
					}
				}

				FileSystem()->CloseFile(FileHandle);
			}
		}

		delete[] ContentServer;
	}

	DeleteInterface(LZMA);

	SignalProgress(MD_RELEASE_EVENT_COMPRESS, MAXSI_PROGRESS_NORMAL, 1.0f);
	SignalProgress(MD_RELEASE_EVENT_UPLOAD, MAXSI_PROGRESS_NORMAL, 1.0f);

	return MAXSI_ERROR_SUCCESS;
}

MaxsiError MD_Release::Release()
{	
	MAXSI_TODO("Call a method at Maxsi Worker to make sure at least N threads are allocated to our purpose!");
	MAXSI_TODO("Use Maxsi Worker for this!");

	ThreadSystem()->CreateThread(ThreadReleaseProduct, (BYTE*)this);

	return MAXSI_ERROR_SUCCESS;
}

MaxsiError MD_Release::ReleaseProduct()
{	
	MAXSI_TODO("Call a method at Maxsi Worker to make sure at least N threads are allocated to our purpose!");
	MAXSI_TODO("Use Maxsi Worker for this!");

	ScanDir(MasterImagePath, _MESTR(""));

	NumBlocks = TotalFileSize / BlockSize;

	ThreadSystem()->CreateThread(ThreadReleaseProductSHA, (BYTE*)this);
	ThreadSystem()->CreateThread(ThreadReleaseProductUpload, (BYTE*)this);

	return MAXSI_ERROR_SUCCESS;
}

MaxsiError MD_Release::ReleaseProductSHA()
{
	CalculateFileHashes();
	CalculateFileList();
	UploadFileList();

	return MAXSI_ERROR_SUCCESS;
}


MaxsiError MD_Release::ReleaseProductUpload()
{
	UploadBlocks();

	return MAXSI_ERROR_SUCCESS;
}

